{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ConvNet-LSTM Stack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we stack an LSTM on top of a convolutional layer to classify IMDB movie reviews by their sentiment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import imdb\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D, LSTM\n",
    "from keras.layers.wrappers import Bidirectional \n",
    "from keras.layers import Conv1D, MaxPooling1D \n",
    "from keras.callbacks import ModelCheckpoint\n",
    "import os\n",
    "from sklearn.metrics import roc_auc_score \n",
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# output directory name:\n",
    "output_dir = 'model_output/cnnLSTM'\n",
    "\n",
    "# training:\n",
    "epochs = 4\n",
    "batch_size = 128\n",
    "\n",
    "# vector-space embedding: \n",
    "n_dim = 64 \n",
    "n_unique_words = 10000 \n",
    "max_review_length = 200 \n",
    "pad_type = trunc_type = 'pre'\n",
    "drop_embed = 0.2 \n",
    "\n",
    "# convolutional layer architecture:\n",
    "n_conv = 64  \n",
    "k_conv = 3 \n",
    "mp_size = 4\n",
    "\n",
    "# LSTM layer architecture:\n",
    "n_lstm = 64 \n",
    "drop_lstm = 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n",
    "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Design neural network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) \n",
    "model.add(SpatialDropout1D(drop_embed))\n",
    "model.add(Conv1D(n_conv, k_conv, activation='relu'))\n",
    "model.add(MaxPooling1D(mp_size))\n",
    "model.add(Bidirectional(LSTM(n_lstm, dropout=drop_lstm)))\n",
    "model.add(Dense(1, activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 200, 64)           640000    \n",
      "_________________________________________________________________\n",
      "spatial_dropout1d_1 (Spatial (None, 200, 64)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, 198, 64)           12352     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1 (None, 49, 64)            0         \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 128)               66048     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 718,529\n",
      "Trainable params: 718,529\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# LSTM layer parameters double due to both reading directions\n",
    "model.summary() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Configure model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n",
    "if not os.path.exists(output_dir):\n",
    "    os.makedirs(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/4\n",
      "25000/25000 [==============================] - 63s - loss: 0.4660 - acc: 0.7604 - val_loss: 0.3090 - val_acc: 0.8739\n",
      "Epoch 2/4\n",
      "25000/25000 [==============================] - 35s - loss: 0.2406 - acc: 0.9045 - val_loss: 0.3007 - val_acc: 0.8709\n",
      "Epoch 3/4\n",
      "25000/25000 [==============================] - 33s - loss: 0.1774 - acc: 0.9350 - val_loss: 0.3929 - val_acc: 0.8492\n",
      "Epoch 4/4\n",
      "25000/25000 [==============================] - 33s - loss: 0.1314 - acc: 0.9544 - val_loss: 0.3887 - val_acc: 0.8527\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f983e8e4588>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 87.4% validation accuracy in epoch 1\n",
    "# slightly higher than bidirectional LSTM and about the same as stacked biLSTM\n",
    "# but epochs are a third as long, or one-sixth as long, respectively\n",
    "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights(output_dir+\"/weights.00.hdf5\") # zero-indexed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000/25000 [==============================] - 29s    \n"
     ]
    }
   ],
   "source": [
    "y_hat = model.predict_proba(x_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE7JJREFUeJzt3X+s3fV93/HnKzgka5vEJlwQss1MVDcLrRTCrsBRpC6N\nM2OgwvwRKqJ1OMiap45V7VZtI9skb5BIZNPGitTSucWridoAZcuwElbmOaBs0yCYkLIARXYIhSsz\n7MaGtENJB33vj/Nxcgz3+p5r33tOLp/nQ7r6fr/v7+f7/X4+3Itf5/vjnJOqQpLUn7dNugOSpMkw\nACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdWjHpDpzM2WefXevWrZt0N6QTffeZ\nwfTd759sP6Q5PPbYY39aVVPztfuRDoB169axf//+SXdDOtF/++hg+vGHJtkLaU5J/mSUdl4CkqRO\nGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkTv1IvxNYkiZp3Y1fntixn7vlyiU/\nhmcAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUqXkDIMn7k3xj6Oe7SX41\nyVlJ9iY50KarWvskuS3JwSRPJLl4aF9bW/sDSbYu5cAkSSc3bwBU1TNVdVFVXQT8deBV4IvAjcC+\nqloP7GvLAJcD69vPduB2gCRnATuAS4FLgB3HQ0OSNH4LvQS0EfhWVf0JsAXY3eq7gavb/Bbgzhp4\nGFiZ5DzgMmBvVR2tqmPAXmDzaY9AknRKFhoA1wJfaPPnVtWLAG16TquvBl4Y2mam1eaqnyDJ9iT7\nk+w/cuTIArsnSRrVyAGQ5EzgKuAP5ms6S61OUj+xULWzqqaranpqamrU7kmSFmghZwCXA1+vqpfa\n8kvt0g5terjVZ4C1Q9utAQ6dpC5JmoCFBMAn+eHlH4A9wPEnebYC9w3Vr2tPA20AXmmXiB4ANiVZ\n1W7+bmo1SdIEjPSFMEl+DPibwN8dKt8C3JNkG/A8cE2r3w9cARxk8MTQ9QBVdTTJzcCjrd1NVXX0\ntEcgSTolIwVAVb0KvPcNte8weCrojW0LuGGO/ewCdi28m5KkxeY7gSWpUwaAJHXKAJCkThkAktQp\nA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIA\nJKlTBoAkdWqkAEiyMsm9Sf44ydNJPpzkrCR7kxxo01WtbZLcluRgkieSXDy0n62t/YEkW+c+oiRp\nqY16BvDrwB9W1V8DPgg8DdwI7Kuq9cC+tgxwObC+/WwHbgdIchawA7gUuATYcTw0JEnjN28AJHk3\n8LPAHQBV9RdV9TKwBdjdmu0Grm7zW4A7a+BhYGWS84DLgL1VdbSqjgF7gc2LOhpJ0shGOQN4H3AE\n+A9JHk/yO0l+HDi3ql4EaNNzWvvVwAtD28+02lx1SdIEjBIAK4CLgdur6kPA/+WHl3tmk1lqdZL6\niRsn25PsT7L/yJEjI3RPknQqRgmAGWCmqh5py/cyCISX2qUd2vTwUPu1Q9uvAQ6dpH6CqtpZVdNV\nNT01NbWQsUiSFmDeAKiq/wO8kOT9rbQReArYAxx/kmcrcF+b3wNc154G2gC80i4RPQBsSrKq3fzd\n1GqSpAlYMWK7XwZ+L8mZwLPA9QzC454k24DngWta2/uBK4CDwKutLVV1NMnNwKOt3U1VdXRRRiFJ\nWrCRAqCqvgFMz7Jq4yxtC7hhjv3sAnYtpIOSpKXhO4ElqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhS\npwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXK\nAJCkTo30ncBJngP+DHgdeK2qppOcBdwNrAOeA36hqo4lCfDrDL4Y/lXgU1X19bafrcA/b7v9TFXt\nXryhvNm6G7+8lLuf03O3XDmR40rSQizkDODnquqiqjr+5fA3Avuqaj2wry0DXA6sbz/bgdsBWmDs\nAC4FLgF2JFl1+kOQJJ2K07kEtAU4/gp+N3D1UP3OGngYWJnkPOAyYG9VHa2qY8BeYPNpHF+SdBpG\nDYAC/muSx5Jsb7Vzq+pFgDY9p9VXAy8MbTvTanPVJUkTMNI9AOAjVXUoyTnA3iR/fJK2maVWJ6mf\nuPEgYLYDnH/++SN2T5K0UCOdAVTVoTY9DHyRwTX8l9qlHdr0cGs+A6wd2nwNcOgk9Tcea2dVTVfV\n9NTU1MJGI0ka2bwBkOTHk7zr+DywCfgmsAfY2pptBe5r83uA6zKwAXilXSJ6ANiUZFW7+bup1SRJ\nEzDKJaBzgS8Onu5kBfD7VfWHSR4F7kmyDXgeuKa1v5/BI6AHGTwGej1AVR1NcjPwaGt3U1UdXbSR\nSJIWZN4AqKpngQ/OUv8OsHGWegE3zLGvXcCuhXdTkrTYfCewJHXKAJCkThkAktQpA0CSOmUASFKn\nDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoA\nkKROGQCS1KmRAyDJGUkeT/KltnxBkkeSHEhyd5IzW/0dbflgW79uaB+fbvVnkly22IORJI1uIWcA\nvwI8PbT8OeDWqloPHAO2tfo24FhV/SRwa2tHkguBa4GfBjYDv5nkjNPrviTpVI0UAEnWAFcCv9OW\nA3wMuLc12Q1c3ea3tGXa+o2t/Rbgrqr6flV9GzgIXLIYg5AkLdyoZwD/DvjHwF+25fcCL1fVa215\nBljd5lcDLwC09a+09j+oz7LNDyTZnmR/kv1HjhxZwFAkSQsxbwAk+XngcFU9NlyepWnNs+5k2/yw\nULWzqqaranpqamq+7kmSTtGKEdp8BLgqyRXAO4F3MzgjWJlkRXuVvwY41NrPAGuBmSQrgPcAR4fq\nxw1vI0kas3nPAKrq01W1pqrWMbiJ+5Wq+lvAg8AnWrOtwH1tfk9bpq3/SlVVq1/bnhK6AFgPfG3R\nRiJJWpBRzgDm8k+Au5J8BngcuKPV7wA+n+Qgg1f+1wJU1ZNJ7gGeAl4Dbqiq10/j+JKk07CgAKiq\nh4CH2vyzzPIUT1V9D7hmju0/C3x2oZ2UJC0+3wksSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUA\nSFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlT8wZA\nkncm+VqSP0ryZJJ/2eoXJHkkyYEkdyc5s9Xf0ZYPtvXrhvb16VZ/JsllSzUoSdL8RjkD+D7wsar6\nIHARsDnJBuBzwK1VtR44Bmxr7bcBx6rqJ4FbWzuSXMjgC+J/GtgM/GaSMxZzMJKk0c0bADXw523x\n7e2ngI8B97b6buDqNr+lLdPWb0ySVr+rqr5fVd8GDjLLl8pLksZjpHsASc5I8g3gMLAX+BbwclW9\n1prMAKvb/GrgBYC2/hXgvcP1WbaRJI3ZSAFQVa9X1UXAGgav2j8wW7M2zRzr5qqfIMn2JPuT7D9y\n5Mgo3ZMknYIVC2lcVS8neQjYAKxMsqK9yl8DHGrNZoC1wEySFcB7gKND9eOGtxk+xk5gJ8D09PSb\nAmI5WHfjlydy3OduuXIix5W0PI3yFNBUkpVt/q8AHweeBh4EPtGabQXua/N72jJt/Veqqlr92vaU\n0AXAeuBrizUQSdLCjHIGcB6wuz2x8zbgnqr6UpKngLuSfAZ4HLijtb8D+HySgwxe+V8LUFVPJrkH\neAp4Dbihql5f3OFIkkY1bwBU1RPAh2apP8ssT/FU1feAa+bY12eBzy68m5KkxeY7gSWpUwaAJHXK\nAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwA\nSeqUASBJnVrQdwJL0iRM6nu23+o8A5CkThkAktSpeQMgydokDyZ5OsmTSX6l1c9KsjfJgTZd1epJ\ncluSg0meSHLx0L62tvYHkmxdumFJkuYzyhnAa8CvVdUHgA3ADUkuBG4E9lXVemBfWwa4HFjffrYD\nt8MgMIAdwKUMvkx+x/HQkCSN37wBUFUvVtXX2/yfAU8Dq4EtwO7WbDdwdZvfAtxZAw8DK5OcB1wG\n7K2qo1V1DNgLbF7U0UiSRragewBJ1gEfAh4Bzq2qF2EQEsA5rdlq4IWhzWZaba76G4+xPcn+JPuP\nHDmykO5JkhZg5ABI8hPAfwR+taq+e7Kms9TqJPUTC1U7q2q6qqanpqZG7Z4kaYFGCoAkb2fwj//v\nVdV/auWX2qUd2vRwq88Aa4c2XwMcOkldkjQBozwFFOAO4Omq+rdDq/YAx5/k2QrcN1S/rj0NtAF4\npV0iegDYlGRVu/m7qdUkSRMwyjuBPwL8beB/J/lGq/1T4BbgniTbgOeBa9q6+4ErgIPAq8D1AFV1\nNMnNwKOt3U1VdXRRRiFJWrB5A6Cq/gezX78H2DhL+wJumGNfu4BdC+mgJGlp+E5gSeqUASBJnTIA\nJKlTBoAkdcoAkKROGQCS1Cm/EewtZJLfmvTcLVdO7NiSTo1nAJLUKQNAkjplAEhSpwwASeqUASBJ\nnfIpIEkjm+STZlp8ngFIUqcMAEnqlAEgSZ0yACSpU94E1qKY1M1BP4JCOnWjfCn8riSHk3xzqHZW\nkr1JDrTpqlZPktuSHEzyRJKLh7bZ2tofSLJ1tmNJksZnlEtAvwtsfkPtRmBfVa0H9rVlgMuB9e1n\nO3A7DAID2AFcClwC7DgeGpKkyZg3AKrqq8DRN5S3ALvb/G7g6qH6nTXwMLAyyXnAZcDeqjpaVceA\nvbw5VCRJY3Sq9wDOraoXAarqxSTntPpq4IWhdjOtNlddOi2TuPdw1/u+A8CGsR95wDdjabEs9k3g\nzFKrk9TfvINkO4PLR5x//vmL1zNpkfkPsZa7U30M9KV2aYc2PdzqM8DaoXZrgEMnqb9JVe2squmq\nmp6amjrF7kmS5nOqAbAHOP4kz1bgvqH6de1poA3AK+1S0QPApiSr2s3fTa0mSZqQeS8BJfkC8FHg\n7CQzDJ7muQW4J8k24Hngmtb8fuAK4CDwKnA9QFUdTXIz8Ghrd1NVvfHGsiRpjOYNgKr65ByrNs7S\ntoAb5tjPLmDXgnonSVoyfhSEJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1\nygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1KmxB0CSzUmeSXIw\nyY3jPr4kaWCsAZDkDOA3gMuBC4FPJrlwnH2QJA2M+wzgEuBgVT1bVX8B3AVsGXMfJEmMPwBWAy8M\nLc+0miRpzFaM+XiZpVYnNEi2A9vb4p8neeYUjnM28KensN1y1uOYYQLj/vAP5n5+nIcd5u+6A/kc\ncOpj/qujNBp3AMwAa4eW1wCHhhtU1U5g5+kcJMn+qpo+nX0sNz2OGfocd49jhj7HvdRjHvcloEeB\n9UkuSHImcC2wZ8x9kCQx5jOAqnotyd8HHgDOAHZV1ZPj7IMkaWDcl4CoqvuB+5f4MKd1CWmZ6nHM\n0Oe4exwz9DnuJR1zqmr+VpKktxw/CkKSOrVsA2C+j5RI8o4kd7f1jyRZN/5eLr4Rxv0PkzyV5Ikk\n+5KM9DjYj7JRPz4kySeSVJK3xJMio4w7yS+03/eTSX5/3H1cbCP8fZ+f5MEkj7e/8Ssm0c/FlGRX\nksNJvjnH+iS5rf03eSLJxYt28Kpadj8MbiB/C3gfcCbwR8CFb2jz94DfavPXAndPut9jGvfPAT/W\n5n9puY97lDG3du8Cvgo8DExPut9j+l2vBx4HVrXlcybd7zGMeSfwS23+QuC5Sfd7Ecb9s8DFwDfn\nWH8F8F8YvI9qA/DIYh17uZ4BjPKREluA3W3+XmBjktneiLaczDvuqnqwql5tiw8zeK/Fcjbqx4fc\nDPwr4Hvj7NwSGmXcfwf4jao6BlBVh8fcx8U2ypgLeHebfw9veB/RclRVXwWOnqTJFuDOGngYWJnk\nvMU49nINgFE+UuIHbarqNeAV4L1j6d3SWehHaWxj8MphOZt3zEk+BKytqi+Ns2NLbJTf9U8BP5Xk\nfyZ5OMnmsfVuaYwy5n8B/GKSGQZPE/7yeLo2UUv2ETpjfwx0kcz7kRIjtlluRh5Tkl8EpoG/saQ9\nWnonHXOStwG3Ap8aV4fGZJTf9QoGl4E+yuBM778n+ZmqenmJ+7ZURhnzJ4Hfrap/k+TDwOfbmP9y\n6bs3MUv2b9lyPQOY9yMlhtskWcHgdPFkp1nLwSjjJsnHgX8GXFVV3x9T35bKfGN+F/AzwENJnmNw\njXTPW+BG8Kh/4/dV1f+rqm8DzzAIhOVqlDFvA+4BqKr/BbyTweflvJWN9P/9qViuATDKR0rsAba2\n+U8AX6l2R2UZm3fc7XLIv2fwj/9yvyYM84y5ql6pqrOral1VrWNw3+Oqqto/me4umlH+xv8zg5v+\nJDmbwSWhZ8fay8U1ypifBzYCJPkAgwA4MtZejt8e4Lr2NNAG4JWqenExdrwsLwHVHB8pkeQmYH9V\n7QHuYHB6eJDBK/9rJ9fjxTHiuP818BPAH7R73s9X1VUT6/RpGnHMbzkjjvsBYFOSp4DXgX9UVd+Z\nXK9Pz4hj/jXgt5P8AwaXQT613F/YJfkCg8t4Z7d7GzuAtwNU1W8xuNdxBXAQeBW4ftGOvcz/20mS\nTtFyvQQkSTpNBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ36/yyDyrVmoQGdAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f983e8e4128>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(y_hat)\n",
    "_ = plt.axvline(x=0.5, color='orange')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'94.61'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
